/**
 * 
 */
package com.gooznt.buscaprop;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * @author gustavo
 *
 */
public class GobyCrawler extends WebCrawler {
	
	private static Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"
			+ "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf"
			+ "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
	
	@Override
	public boolean shouldVisit(WebURL url) {
		
		String href = url.getURL().toLowerCase();
		
		if (isDetailsLink(href) && CrawlerExecutor.detailsFound <= CrawlerExecutor.getMaxDetailsFound()){
			synchronized (CrawlerExecutor.detailsFound) {
				CrawlerExecutor.detailsFound++;
			}
		}
		
		if (filters.matcher(href).matches()
				|| CrawlerExecutor.detailsFound > CrawlerExecutor.getMaxDetailsFound()) {
			return false;
		}
		if (href.startsWith(CrawlerExecutor.getDomain()) && (isDetailsLink(url.getURL().toLowerCase()))) {
			return true;
		}
		
		return false;
	}
	
	@Override
	public void visit(Page page) {
		System.out.println(page.getWebURL().getURL());
		if (isDetailsLink(page.getWebURL().getURL().toLowerCase())) {
			synchronized (CrawlerExecutor.fw) {
				try {
					System.out.println("Match: " + page.getWebURL().getURL());
					CrawlerExecutor.fw.write(page.getWebURL().getURL().toLowerCase() + "\n");
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		
		
	}
	
	/**
	 * Method redefined by a crawler to check if the page is a details page or not
	 * @param lowerCase the page url
	 * @return true if the page is a details link
	 */
	public boolean isDetailsLink(String lowerCaseURL) {
		Matcher m = CrawlerExecutor.getDetailsLinkPattern().matcher(lowerCaseURL);
		return m.find();
	}
	
	@Override
	public void onBeforeExit() {
		synchronized (CrawlerExecutor.fw) {
			try {
				CrawlerExecutor.fw.close();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}
}
